{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 准备工作"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 前序数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['../data_base/knowledge_path/VMAX-S\\\\ZXVMAX-S（V6.20.80.02）告警处理.pdf', '../data_base/knowledge_path/VMAX-S\\\\ZXVMAX-S（V6.20.80.02）故障管理概述.pdf', '../data_base/knowledge_path/VMAX-S\\\\ZXVMAX-S（V6.23）产品描述（5GC业务）.pdf', '../data_base/knowledge_path/VMAX-S\\\\ZXVMAX-S（V6.23）产品描述（上网日志业务）.pdf', '../data_base/knowledge_path/VMAX-S\\\\ZXVMAX-S（V6.23）产品描述（数据业务）.pdf', '../data_base/knowledge_path/VMAX-S\\\\ZXVMAX-S（V6.23）产品描述（端到端业务）.pdf', '../data_base/knowledge_path/VMAX-S\\\\ZXVMAX-S（V6.23）产品描述（语音业务）.pdf']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\learn-llm-rag-easily\\.venv\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "切分后的文件数量：891\n",
      "切分后的字符数（可以用来大致评估 token 数）：319269\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"ZXVMAX-S多维价值分析系统告警处理\\n4.修改配置项'Agent的Java堆栈大小'的参数值，重启已修正的flume角色实例；\\n5.等待5分钟，检查告警是否已恢复。如果恢复则结束，如果没恢复则进行第6步；\\n6.以root用户登录产生该告警的服务器；\\n7.执行ps-ef|grepFlumeAgent命令，获取flume服务（FlumeAgent进程）的PID；\\n8.执行jmap-heap<PID>命令，检查flume服务（FlumeAgent进程）'Agent的Java堆栈大小'配置是否生效（蓝色字体的值）。如果不生效则跳到第10步，如果生效则进行第9步；\\n9.等待5分钟，检查告警是否已恢复。如果恢复则结束，如果没恢复则进行第10步；\\n10.备案，等待5分钟，执行步骤8，查看flume进程的堆内存配置是否生效。生效则结束，不生效则通知上级维护人员进一步排查。\\n1.238124010036FlumeAgent进程5分钟意外退出次数告警描述\""
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#!/usr/bin/env python\n",
    "# coding: utf-8\n",
    "\n",
    "# 批量处理文件夹中所有文件\n",
    "import os\n",
    "\n",
    "# 获取folder_path下所有文件路径，储存在file_paths里\n",
    "file_paths = []\n",
    "folder_path = '../data_base/knowledge_path/VMAX-S'\n",
    "for root, dirs, files in os.walk(folder_path):\n",
    "    for file in files:\n",
    "        file_path = os.path.join(root, file)\n",
    "        file_paths.append(file_path)\n",
    "print(file_paths)\n",
    "\n",
    "from langchain.document_loaders.pdf import PyMuPDFLoader\n",
    "# from langchain.document_loaders.markdown import UnstructuredMarkdownLoader\n",
    "\n",
    "# 遍历文件路径并把实例化的loader存放在loaders里\n",
    "loaders = []\n",
    "\n",
    "for file_path in file_paths:\n",
    "    file_type = file_path.split('.')[-1]\n",
    "    if file_type == 'pdf':\n",
    "        loaders.append(PyMuPDFLoader(file_path))\n",
    "    else:\n",
    "        print(f\"Unsupported file type: {file_type} for file {file_path}\")\n",
    "\n",
    "# 下载文件并存储到text\n",
    "# 加载所有文档内容到 texts\n",
    "texts = []\n",
    "for loader in loaders:\n",
    "    texts.extend(loader.load())  # 关键步骤：初始化 texts\n",
    "\n",
    "    \n",
    "# 作数据清洗\n",
    "# 修改后的数据清洗部分（替换原始代码中对应段落）\n",
    "import re\n",
    "\n",
    "# 预编译正则表达式（提升效率）\n",
    "linebreak_pattern = re.compile(\n",
    "    r'(?<![\\\\u4e00-\\\\u9fff])\\n(?![\\\\u4e00-\\\\u9fff])',  # 负向断言匹配非中文环境换行\n",
    "    flags=re.DOTALL\n",
    ")\n",
    "space_pattern = re.compile(r'[ 　]+')  # 匹配半角/全角空格\n",
    "special_chars = ['•', '▪', '▫', '▶', '®', '©']  # 可扩展的干扰符号列表\n",
    "\n",
    "# 替换原始代码中的清洗循环\n",
    "for text in texts:\n",
    "    # 1. 清理非中文环境换行\n",
    "    text.page_content = re.sub(\n",
    "        linebreak_pattern,\n",
    "        lambda m: m.group().replace('\\n', ''),\n",
    "        text.page_content\n",
    "    )\n",
    "\n",
    "    # 2. 批量清理特殊符号\n",
    "    for char in special_chars:\n",
    "        text.page_content = text.page_content.replace(char, '')\n",
    "\n",
    "    # 3. 安全删除空格（保留URL等特殊场景）\n",
    "    text.page_content = space_pattern.sub('', text.page_content)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#导入文本分割器\n",
    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
    "''' \n",
    "* RecursiveCharacterTextSplitter 递归字符文本分割\n",
    "RecursiveCharacterTextSplitter 将按不同的字符递归地分割(按照这个优先级[\"\\n\\n\", \"\\n\", \" \", \"\"])，\n",
    "    这样就能尽量把所有和语义相关的内容尽可能长时间地保留在同一位置\n",
    "RecursiveCharacterTextSplitter需要关注的是4个参数：\n",
    "\n",
    "* separators - 分隔符字符串数组\n",
    "* chunk_size - 每个文档的字符数量限制\n",
    "* chunk_overlap - 两份文档重叠区域的长度\n",
    "* length_function - 长度计算函数\n",
    "'''\n",
    "\n",
    "\n",
    "# 知识库中单段文本长度\n",
    "CHUNK_SIZE = 512\n",
    "\n",
    "# 知识库中相邻文本重合长度\n",
    "OVERLAP_SIZE = 50\n",
    "\n",
    "\n",
    "# 使用递归字符文本分割器\n",
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=CHUNK_SIZE,\n",
    "    chunk_overlap=OVERLAP_SIZE\n",
    ")\n",
    "\n",
    "\n",
    "split_docs = text_splitter.split_documents(texts)\n",
    "print(f\"切分后的文件数量：{len(split_docs)}\")\n",
    "\n",
    "\n",
    "\n",
    "print(f\"切分后的字符数（可以用来大致评估 token 数）：{sum([len(doc.page_content) for doc in split_docs])}\")\n",
    "\n",
    "\n",
    "\n",
    "split_docs[300].page_content\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 前序embedding模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ZhipuAIEmbeddings模型\n",
    "# import os\n",
    "# from dotenv import load_dotenv, find_dotenv\n",
    "\n",
    "# _ = load_dotenv(find_dotenv())\n",
    "\n",
    "# from langchain_community.embeddings import ZhipuAIEmbeddings\n",
    "\n",
    "# my_emb = ZhipuAIEmbeddings(\n",
    "#     model=\"embedding-2\",\n",
    "#     api_key = os.environ['ZHIPUAI_API_KEY'],\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from langchain_ollama.embeddings import OllamaEmbeddings\n",
    "\n",
    "# # 初始化嵌入模型\n",
    "# my_emb = OllamaEmbeddings(\n",
    "#     base_url='http://localhost:11434',\n",
    "#     model=\"bge-m3:latest\"\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_community.embeddings import XinferenceEmbeddings\n",
    "\n",
    "# 使用API代理服务提高访问稳定性\n",
    "my_emb = XinferenceEmbeddings(\n",
    "    server_url=\"http://120.79.252.32:9997\", \n",
    "    model_uid=\"my_qwen3_embed_0.6b\"\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 构建Milvus向量库"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from langchain_community.vectorstores import Milvus\n",
    "from langchain_core.documents import Document\n",
    "\n",
    "\n",
    "# 创建嵌入模型\n",
    "my_emb = XinferenceEmbeddings(\n",
    "    server_url=\"http://120.79.252.32:9997\", \n",
    "    model_uid=\"my_qwen3_embed_0.6b\"\n",
    ")\n",
    "\n",
    "# 向量库创建\n",
    "# connection_args = {\n",
    "#     \"uri\": \"tcp://120.79.252.32:19530\"\n",
    "# }\n",
    "\n",
    "connection_args = {\n",
    "    \"host\": \"120.79.252.32\",\n",
    "    \"port\": 19530,\n",
    "}\n",
    "\n",
    "# 定义每批处理的文档数量\n",
    "batch_size = 30\n",
    "\n",
    "# 如果只想导入部分数据\n",
    "# split_docs = split_docs[:3]\n",
    "try:\n",
    "    # 计算总批次数\n",
    "    total_batches = (len(split_docs) + batch_size - 1) // batch_size\n",
    "    \n",
    "    # 初始化向量数据库（如果是第一次创建）\n",
    "    vectordb = None\n",
    "    \n",
    "    for batch_num in range(total_batches):\n",
    "        # 计算当前批次的起始和结束索引\n",
    "        start_idx = batch_num * batch_size\n",
    "        end_idx = min((batch_num + 1) * batch_size, len(split_docs))\n",
    "        \n",
    "        # 获取当前批次的文档\n",
    "        batch_docs = split_docs[start_idx:end_idx]\n",
    "        \n",
    "        print(f\"正在处理第 {batch_num + 1}/{total_batches} 批文档 (文档 {start_idx}-{end_idx-1})\")\n",
    "\n",
    "        if batch_num == 0:\n",
    "            # 第一次创建向量数据库\n",
    "            vectordb = Milvus.from_documents(\n",
    "            documents=batch_docs,\n",
    "            embedding=my_emb,\n",
    "            collection_name=\"ZXVMAXS1121\",\n",
    "            drop_old=False,\n",
    "            connection_args=connection_args,\n",
    "            )\n",
    "            \n",
    "            # 如果使用Milvus的混合检索\n",
    "            # 创建 Milvus VectorStore，实现 dense + sparse 混合检索 \n",
    "            # vectordb = Milvus.from_documents( \n",
    "            # documents=batch_docs, \n",
    "            # embedding=my_emb, # 用于语义检索的 dense 向量 \n",
    "            # builtin_function=BM25BuiltInFunction(), # BM25 的 sparse 全文检索 \n",
    "            # vector_field=[\"dense\", \"sparse\"], # 指定两个向量字段名称 \n",
    "            # connection_args=connection_args, \n",
    "            # collection_name=\"ZXVMAXS6\",\n",
    "            # consistency_level=\"Strong\", \n",
    "            # drop_old=True, # 若已有旧 collection，可删除重建 \n",
    "            # )\n",
    "\n",
    "        else:\n",
    "            # 后续批次添加到现有集合\n",
    "            vectordb.add_documents(batch_docs)\n",
    "        \n",
    "        # 每批处理后持久化\n",
    "        # vectordb.persist()\n",
    "        print(f\"第 {batch_num + 1} 批文档已成功导入并持久化\")\n",
    "    \n",
    "    print(\"所有文档已成功导入并持久化到向量数据库。\")\n",
    "    \n",
    "except Exception as e:\n",
    "    print(f\"处理过程中发生错误: {e}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 连接数据库进行测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\will\\AppData\\Local\\Temp\\ipykernel_40932\\81257760.py:17: LangChainDeprecationWarning: The class `Milvus` was deprecated in LangChain 0.2.0 and will be removed in 1.0. An updated version of the class exists in the :class:`~langchain-milvus package and should be used instead. To use it run `pip install -U :class:`~langchain-milvus` and import as `from :class:`~langchain_milvus import MilvusVectorStore``.\n",
      "  vectordb = Milvus(\n"
     ]
    }
   ],
   "source": [
    "from langchain_community.vectorstores import Milvus\n",
    "\n",
    "my_emb = XinferenceEmbeddings(\n",
    "    server_url=\"http://120.79.252.32:9997\", \n",
    "    model_uid=\"my_qwen3_embed_0.6b\"\n",
    ")\n",
    "\n",
    "# connection_args = {\n",
    "#     \"uri\": \"tcp://120.79.252.32:19530\"\n",
    "# }\n",
    "\n",
    "connection_args = {\n",
    "    \"host\": \"120.79.252.32\",\n",
    "    \"port\": 19530,\n",
    "}\n",
    "# Milvus 连接参数\n",
    "vectordb = Milvus(\n",
    "        embedding_function=my_emb,\n",
    "        collection_name=\"ZXVMAXS\",  # Milvus 集合名称\n",
    "        connection_args=connection_args,\n",
    "    )\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "所有Collection列表:\n",
      "1. ZXVMAXS\n",
      "2. ZXVMAXS1121\n",
      "总共找到 2 个collection\n"
     ]
    }
   ],
   "source": [
    "from pymilvus import connections, utility\n",
    "\n",
    "def list_all_collections():\n",
    "    \"\"\"获取所有collection列表\"\"\"\n",
    "    try:\n",
    "        # 连接到Milvus（使用与vectordb相同的参数）\n",
    "        connections.connect(\n",
    "            alias=\"default\",\n",
    "            host=\"120.79.252.32\",\n",
    "            port=\"19530\"\n",
    "        )\n",
    "        \n",
    "        # 获取所有collection\n",
    "        collections = utility.list_collections()\n",
    "        print(\"所有Collection列表:\")\n",
    "        for i, coll_name in enumerate(collections, 1):\n",
    "            print(f\"{i}. {coll_name}\")\n",
    "            \n",
    "        return collections\n",
    "        \n",
    "    except Exception as e:\n",
    "        print(\"获取collection列表失败:\", e)\n",
    "        return []\n",
    "    finally:\n",
    "        # 断开连接\n",
    "        connections.disconnect(\"default\")\n",
    "\n",
    "# 执行\n",
    "collections = list_all_collections()\n",
    "print(f\"总共找到 {len(collections)} 个collection\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "相似性搜索结果:\n",
      "结果 1: 1.106113030103thrift进程文件描述符使用百分比............................60\n",
      "1.107113030106thrift进程每分钟垃圾回收时间所占的百分比..................61\n",
      "1.108113030107thrift进程5分钟意外退出次数...............................61\n",
      "1.109113040111thrift2进程文件描述符使用百分比...........................61\n",
      "1.110113040114thrift2进程每分钟垃圾回收时间所占的百分比.................62\n",
      "1.111113040115thrift2进程5分钟意外退出次数..............................62\n",
      "1.112113040117RegionServer进程客户端请求时延10毫秒内次数占比............63\n",
      "1.113113040118RegionServer进程客户端请求时延2000毫秒内次数占比..........63 - 元数据: {'pk': 461578465653297760, 'producer': 'Apache FOP Version 2.3', 'creator': 'DITA Open Toolkit', 'creationdate': '2022-06-23T16:34:22+08:00', 'source': '../data_base/knowledge_path/VMAX-S/ZXVMAX-S（V6.20.80.02）告警处理.pdf', 'file_path': '../data_base/knowledge_path/VMAX-S/ZXVMAX-S（V6.20.80.02）告警处理.pdf', 'total_pages': 330, 'format': 'PDF 1.4', 'title': '目录', 'author': '', 'subject': '', 'keywords': '', 'moddate': '', 'trapped': '', 'modDate': '', 'creationDate': \"D:20220623163422+08'00'\", 'page': 5}\n",
      "结果 2: ZXVMAX-S多维价值分析系统告警处理可能原因确保实际填充率达到预期填充率处理步骤确保实际填充率达到预期填充率\n",
      "7.2194000200303HTTP接口下行TCP重传报文数完整率(小时)告警描述确保实际填充率达到预期填充率告警级别警告可能原因确保实际填充率达到预期填充率处理步骤确保实际填充率达到预期填充率\n",
      "7.2204000200304HTTP接口TCP建链响应时延（ms）完整率(小时)告警描述确保实际填充率达到预期填充率告警级别警告可能原因确保实际填充率达到预期填充率处理步骤确保实际填充率达到预期填充率\n",
      "7.2214000200305HTTP接口TCP建链确认时延（ms）完整率(小时)告警描述确保实际填充率达到预期填充率\n",
      "276\n",
      "SJ-20220623151803-011|2022-06-20（R1.0） - 元数据: {'pk': 461578465653298261, 'producer': 'Apache FOP Version 2.3', 'creator': 'DITA Open Toolkit', 'creationdate': '2022-06-23T16:34:22+08:00', 'source': '../data_base/knowledge_path/VMAX-S/ZXVMAX-S（V6.20.80.02）告警处理.pdf', 'file_path': '../data_base/knowledge_path/VMAX-S/ZXVMAX-S（V6.20.80.02）告警处理.pdf', 'total_pages': 330, 'format': 'PDF 1.4', 'title': '目录', 'author': '', 'subject': '', 'keywords': '', 'moddate': '', 'trapped': '', 'modDate': '', 'creationDate': \"D:20220623163422+08'00'\", 'page': 297}\n"
     ]
    }
   ],
   "source": [
    "# 测试相似性搜索\n",
    "query = \"ZXVMAXS的5G上网日志有那些功能？\"\n",
    "results = vectordb.similarity_search(query, k=2)\n",
    "print(\"相似性搜索结果:\")\n",
    "for i, doc in enumerate(results):\n",
    "    print(f\"结果 {i+1}: {doc.page_content} - 元数据: {doc.metadata}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "带分数的搜索结果:\n",
      "内容: 1.106113030103thrift进程文件描述符使用百分比............................60\n",
      "1.107113030106thrift进程每分钟垃圾回收时间所占的百分比..................61\n",
      "1.108113030107thrift进程5分钟意外退出次数...............................61\n",
      "1.109113040111thrift2进程文件描述符使用百分比...........................61\n",
      "1.110113040114thrift2进程每分钟垃圾回收时间所占的百分比.................62\n",
      "1.111113040115thrift2进程5分钟意外退出次数..............................62\n",
      "1.112113040117RegionServer进程客户端请求时延10毫秒内次数占比............63\n",
      "1.113113040118RegionServer进程客户端请求时延2000毫秒内次数占比..........63, 分数: 6436.9302\n",
      "内容: ZXVMAX-S多维价值分析系统告警处理可能原因确保实际填充率达到预期填充率处理步骤确保实际填充率达到预期填充率\n",
      "7.2194000200303HTTP接口下行TCP重传报文数完整率(小时)告警描述确保实际填充率达到预期填充率告警级别警告可能原因确保实际填充率达到预期填充率处理步骤确保实际填充率达到预期填充率\n",
      "7.2204000200304HTTP接口TCP建链响应时延（ms）完整率(小时)告警描述确保实际填充率达到预期填充率告警级别警告可能原因确保实际填充率达到预期填充率处理步骤确保实际填充率达到预期填充率\n",
      "7.2214000200305HTTP接口TCP建链确认时延（ms）完整率(小时)告警描述确保实际填充率达到预期填充率\n",
      "276\n",
      "SJ-20220623151803-011|2022-06-20（R1.0）, 分数: 6496.2637\n",
      "内容: ZXVMAX-S多维价值分析系统告警处理告警级别警告可能原因确保实际填充率达到预期填充率处理步骤确保实际填充率达到预期填充率\n",
      "7.2634000200347HTTP接口窗口大小合规率(小时)告警描述确保实际填充率达到预期填充率告警级别警告可能原因确保实际填充率达到预期填充率处理步骤确保实际填充率达到预期填充率\n",
      "7.2644000200348HTTP接口MSS大小合规率(小时)告警描述确保实际填充率达到预期填充率告警级别警告可能原因确保实际填充率达到预期填充率处理步骤确保实际填充率达到预期填充率\n",
      "292\n",
      "SJ-20220623151803-011|2022-06-20（R1.0）, 分数: 6539.9897\n"
     ]
    }
   ],
   "source": [
    "# 获取带相似度分数的结果\n",
    "results_with_score = vectordb.similarity_search_with_score(query, k=3)\n",
    "print(\"带分数的搜索结果:\")\n",
    "for doc, score in results_with_score:\n",
    "    print(f\"内容: {doc.page_content}, 分数: {score:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (learn-llm-rag-easily)",
   "language": "python",
   "name": "learn-llm-rag-easily"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
